'use strict';
// var https = require('https');
var http = require('http');
var https = require('https');
var fs = require('fs');
var path = require('path');
var cheerio = require('cheerio');
var _ = require('lodash');
var {URL} = require('url');
var axios = require('axios');
async function wgrep (env, url){
    var linkspool = [];
    var resourcespool = [];
    let desturl = null;
    var opts = null;
    var protocal = 'http'
    if (url.substr(0, 5) === 'https'){
        desturl = new URL(url);
        opts = {
            hostname: desturl.hostname,
            port: desturl.port === '' ? 443 : desturl.port,
            path: desturl.pathname,
            agent: false  // Create a new agent just for this one request
        
        }
        protocal = 'https';
    } else if (url.substr(0, 4) === 'http'){
        desturl = new URL(url);
        opts = {
            hostname: desturl.hostname,
            port: desturl.port === '' ? 80 : desturl.port,
            path: desturl.pathname,
            agent: false  // Create a new agent just for this one request
        
        }
    } else {
        desturl = new URL('http://' + url);
        opts = {
            hostname: desturl.hostname,
            port: desturl.port === '' ? 80 : desturl.port,
            path: desturl.pathname,
            agent: false  // Create a new agent just for this one request
        
        }
    }
    await grepTextPage(desturl.pathname).catch((err) => console.error(err));
    function grepTextPage(url){
        // let pageurl = nexturl ? path.join(currenturl, nexturl) : currenturl;
        if (linkspool.indexOf(url) == -1){
            linkspool.push(url)
        } else {
            //已经爬过该路径或者是文件连接
            return new Promise((resolve) => {});
        }
        let reg = /\.png|\.jpg|\.gif|\.jpeg|\.pdf|\.rar|\.zip/ig;
        if (reg.test(url)){
            console.log(`非页面连接${url}`)
            return new Promise((resolve) => {});
        }
        console.log(`<=============================页面分析开始${url}`);
        let pageObj = _.assign({}, opts, {
            path:url
        });
        return new Promise( (resolve, reject) => {
            getHttpClient().get(pageObj, (res) => {
                // Do stuff
                // console.log(res);
                res.setEncoding('utf-8');
                let context = "";
                res.on('data', (clunk) => {
                    context += clunk;
                });
                res.on('end', async ()=>{
                    let regexes = regGet();
                    // let result = html.match(/["'][^"']+["']/g);
                    let resource = grepImages(context);
                    let pages = grepCss(context).concat(grepJS(context), grepLinks(context));
                    resource.forEach( async (resourceurl) => {
                        if (resourceurl.substr(0, 4) === 'http' 
                        || resourceurl.substr(0, 11) === 'javascript:' 
                        || resourceurl.substr(0, 2) === '${'
                        || resourceurl.substr(0, 1) === '#'
                        ){}
                        else if (resourceurl.substr(0, 1) === '/') {
                            await grepResources(resourceurl);
                        } else {
                            let webpath = path.dirname(url);
                            webpath = path.join(webpath, resourceurl);
                            await grepResources(webpath);
                        }
                        
                    });
                    console.log(`页面分析完毕${url}\n=============================>`);
                    console.log(`内联页面分析...`);
                    pages.forEach(async (page) => {
                        if (page.substr(0, 4) === 'http' 
                        || page.substr(0, 11) === 'javascript:' 
                        || page.substr(0, 7) === 'mailto:' 
                        || page.substr(0, 2) === '${'
                        || page.substr(0, 1) === '#'
                        ){
                            //去除href中的http外链，javascript方法及${}变量
                        } else if (page.substr(0, 1) === '/') {
                            await grepTextPage(page).catch((err) => console.error(err));;
                        } else {
                            let webpath = path.dirname(url);
                            webpath = path.join(webpath, page);
                            await grepTextPage(webpath).catch((err) => console.error(err));;
                        }
                        
                    });
                    
                    let localpath = path.join(env.cwd + url);
                    let dirname = path.dirname(url);
                    let filename = path.basename(url);
                    console.log('filename @ pagelink:' + filename);
                    if (filename.indexOf('.') == -1) {
                        localpath = path.join(localpath, 'index.html');
                        dirname = path.dirname(path.join(url, 'index.html'));
                        // dirname = path.dirname(url);
                    }
                    if (!fs.existsSync(dirname)) {
                        mkdirsSync(dirname);
                    }
                    fs.writeFileSync(localpath, context,{flag:'w+',encoding:'utf-8',mode:'0666'});
                    resolve(context);
                    
                });
                
            }).on('error', function(err) {
                // console.log(err);
                reject(err);
            });
        });
        
    }
    async function grepResources(url){
        //清除/../的情况
        url = decodeURI(url.replace(/^\/\.\./, ""));
        let resourcelink = path.join(opts.hostname, url);
        if (resourcespool.indexOf(resourcelink) == -1){
            resourcespool.push(resourcelink)
        } else {
            //已经爬过该路径
            return;
        }
        // url = escape(url);
        let localpath = path.join(env.cwd + url);
        let dirname = path.dirname(url);
        let filename = path.basename(url);
        
        let encodeUrl = encodeURI(resourcelink);
        if (fs.existsSync(localpath)){
            console.log(`${localpath} exited`);
            return;
        } 
        await axios({
            method: 'get',
            url: `${protocal}://${encodeUrl}`,
            responseType: 'stream',
          })
        .then(function (response) {
            
            if (!fs.existsSync(dirname)) {
                mkdirsSync(dirname);
            }
            let writer = fs.createWriteStream(localpath)
            response.data.pipe(writer);
            return new Promise((resolve, reject) => {
                writer.on("finish", resolve);
                writer.on("error", reject);
            });
        }).catch((err) => {

            // console.error(localpath);
            // console.error(dirname);
            // console.error(url);
            // console.error(encodeUrl);
            console.error(`axios get error @ link:${resourcelink}`);
        })
        // console.log(resourcelink);
    }
    
    function grepCss(html){
        console.log(`开始分析css链接`);
        let result = html.match(/\/[^"']+(\.css)/ig);
        // console.log(`css链接数组\n${result}`);
        return result || [];
    }
    function grepJS(html){
        console.log(`开始分析js链接`);
        let result = html.match(/\/[^"']+(\.js)/ig);
        // console.log(`js链接数组\n${result}`);
        return result || [];
    }
    function grepImages(html){
        console.log(`开始分析images链接`);
        let reg1 = /"([^"']+(\.png|\.jpg|\.gif|\.jpeg))"/ig;
        let reg2 = /url\(([^\)]+(\.png|\.jpg|\.gif|\.jpeg))\)/ig;
        let result = [];
        let res;
        while(res = reg1.exec(html)){
            result.push(res);
        }
        while(res = reg2.exec(html)){
            result.push(res);
        }
        let urls = _.map(result, (item) => item[1]);
        // console.log(`images链接数组\n${result}`);
        return urls;
    }
    function grepLinks(html){
        console.log(`开始分析page链接`);
        let result = [];
        let reg = /\<a[^\>]+href="([^"]*)"/ig;
        let res;
        while(res = reg.exec(html)){
            result.push(res);
        }
        let urls = _.map(result, (item) => item[1]);
        // console.log(`pages链接数组\n${urls}`);
        return urls;
    }
    function getHttpClient(){
        return protocal == 'http' ? http : https;
    }
    function regGet(){
        return [
        ]
    }
};

function mkdirsSync (dirpath, mode) { 
    try
    {
        if (!fs.existsSync(dirpath)) {
            let pathtmp;
            dirpath.split(/[/\\]/).forEach(function (dirname) {  //这里指用/ 或\ 都可以分隔目录  如  linux的/usr/local/services   和windows的 d:\temp\aaaa
                if (dirname == '') return;
                if (pathtmp) {
                    pathtmp = path.join(pathtmp, dirname);
                }
                else {
                    pathtmp = dirname;
                }
                if (!fs.existsSync(pathtmp)) {
                    console.log('make dir:' + pathtmp);
                    if (!fs.mkdirSync(pathtmp, mode)) {
                        return false;
                    }
                }
            });
        }
        return true; 
    }catch(e)
    {
        console.error("create director fail! path=" + dirpath +" errorMsg:" + e);        
        return false;
    }
}
module.exports = wgrep;